library(here)
setwd(here())

library(haven)
library(reshape2)
library(mellonMisc)
library(dplyr)
library(glmnet)
library(ridge)
library(brms)
library(readxl)
library(metRology)
library(survey)
library(plm)
library(plotly)
library(ggplot2)
library(mellonMisc)
library(gridExtra)
library(cmdstanr)

source("scripts/promise_functions.R")

#### Formatting #### 

promise <- read_spss("data/raw/FullFact_May18_Client.sav")


## ceilign and floor effects
ceilings1 <- rowSums(promise[, c("importgrid1", "importgrid2", "importgrid3")]==11)
floors1 <- rowSums(promise[, c("importgrid1", "importgrid2", "importgrid3")]==1)
ceilings2 <- rowSums(promise[, c("importgrid4", "importgrid5", "importgrid6")]==11)
floors2 <- rowSums(promise[, c("importgrid4", "importgrid5", "importgrid6")]==1)


p2.vars <- grepl("p2", colnames(promise))
colnames(promise)[p2.vars] <- gsub("g1", "g4", colnames(promise)[p2.vars])
colnames(promise)[p2.vars] <- gsub("g2", "g5", colnames(promise)[p2.vars])
colnames(promise)[p2.vars] <- gsub("g3", "g6", colnames(promise)[p2.vars])
colnames(promise) <- gsub("p2$|p1$", "", colnames(promise))
colnames(promise) <- gsub("grid1", "g1", colnames(promise))
colnames(promise) <- gsub("grid2", "g2", colnames(promise))
colnames(promise) <- gsub("grid3", "g3", colnames(promise))
colnames(promise) <- gsub("grid4", "g4", colnames(promise))
colnames(promise) <- gsub("grid5", "g5", colnames(promise))
colnames(promise) <- gsub("grid6", "g6", colnames(promise))


promise.parts <- split(dtf(promise), 1:nrow(promise))
promise.outs <- lapply(promise.parts, rowToCol)

promise.total <- bind_rows(promise.outs)
promise.total$pageid <- paste0(promise.total$id, "_", promise.total$page)

promise.total[, grepl("issue", colnames(promise.total))] <- 
  sapply(promise.total[, grepl("issue", colnames(promise.total))], unlist)
promise.total$approval <- unlist(promise.total$approval)
promise.total$import <- unlist(promise.total$import)
promise.total <- dtf(promise.total)

all.iss.vars <- colnames(promise.total)[grepl("issue", colnames(promise.total))]

prom.tot <- apply(promise.total[, all.iss.vars], 1, function(x) which(x==1))
prom.tot <- t(prom.tot)

colnames(prom.tot) <- c("g1", "g2", "g3", "g4")

prom.sum <- dtf(import = promise.total$import,
                approval = promise.total$approval,
                prom.tot, id= promise.total$id, page = promise.total$pageid)




pf <- read_excel("data/raw/Post-Feedback Snippets V3.xlsx")

pf$id <- 1:nrow(pf)
snip.agree <- read_excel("data/raw/snippetIDsMII.agreed.xlsx")

pf$agreed_cat <- snip.agree$agreed_cat[match(pf$Snippets, snip.agree$Snippets)]
pf$agreed_cat_string <- pf$agreed_cat
pf$agreed_cat <- as.numeric(factor(pf$agreed_cat))
pf$Snippets[pf$Snippets=="Keep the Trident nuclear deterrant" ] <-  "Keep the Trident nuclear deterrent" 
pf$Snippets[pf$Snippets== "Create a 'Veterans Board' to co-ordinate treatment of military veterans" ] <-   "Create a 'Veterans Board' to coordinate treatment of military veterans"


pfstatus <- read_excel('data/raw/Update Final with weights.xlsx')
pfstatus <- pfstatus %>% rnm(status = `Current Status`)
pfstatus$`Promise name` <- gsub("\n", "", pfstatus$`Promise name`)

pfstatus$statuscomb <- NA
pfstatus$status[pfstatus$status %in% c("Partially")] <- 'Partially Fulfilled'
pfstatus$status[pfstatus$status %in% c("Fully Fuliflled", "RECHECK (Fully Fulfilled)")] <- 'Fully Fulfilled'
pfstatus$status[pfstatus$status %in% c("UKNOWN", "NA")] <- NA

pfstatus$statuscomb[pfstatus$status %in% c("Pending", "Broken")] <- 'Not fulfilled'
pfstatus$statuscomb[pfstatus$status %in% c("Fully Fulfilled", "Partially Fulfilled")] <- 'Fulfilled'

cppg <- read_excel("data/raw/validforcppg.xlsx")

pfstatus$cppg <- cppg$cppg[match(pfstatus$ID, cppg$ID)]

pf$statuscomb <- pfstatus$statuscomb[match(pf$Snippets, pfstatus$Snippets)]
pf$status_quo <- pfstatus$status_quo[match(pf$Snippets, pfstatus$Snippets)]
pf$cppg <- pfstatus$cppg[match(pf$Snippets, pfstatus$Snippets)]
pf$cppg <- pf$cppg==1
mii.lookup <- unique(dtf(cat = pf$agreed_cat_string, code = pf$agreed_cat)) %>% arrange(code)

save(mii.lookup, file = "data/intermediate/mii.lookup.rda")


library(dplyr)
library(mellonMisc)


prop.table(table(snip.agree$MII_cat1==snip.agree$Cat1))
prop.table(table(snip.agree$MII_cat1==snip.agree$agreed_cat))
prop.table(table(snip.agree$Cat1==snip.agree$agreed_cat))

rm(snip.agree)

# agreements between coders:

prom.sum$mii1 <- pf$agreed_cat[match(prom.sum$g1, pf$id)]
prom.sum$mii2 <- pf$agreed_cat[match(prom.sum$g2, pf$id)]
prom.sum$mii3 <- pf$agreed_cat[match(prom.sum$g3, pf$id)]
prom.sum$mii4 <- pf$agreed_cat[match(prom.sum$g4, pf$id)]

all(prom.sum$id %in% promise$ID)
all(promise$ID %in% prom.sum$id)

#### Format and merge outcomes #### 
library(here)
library(readxl)
# all(pf$Snippets %in% snip.agree$Snippets)
# all(snip.agree$Snippets %in% pf$Snippets)
save(pf, file = "data/intermediate/pf.rda")


library(brms)

# What priors are needed?

get_prior(import ~ (1  |mm(g1, g2, g3, g4)) + (1|mm(mii1, mii2, mii3, mii4)) + (1|id) + (1|page), 
          data = prom.sum)

prom.sum$vote17  <- promise$Vote2017GE[match(prom.sum$id, promise$ID)]

all(tapply(prom.sum$vote17, prom.sum$id, sd)==0)

prom.sum$g1.c <- prom.sum$g1
prom.sum$g2.c <- prom.sum$g2
prom.sum$g3.c <- prom.sum$g3
prom.sum$g4.c <- prom.sum$g4

prom.sum$g1.c[prom.sum$vote17!=1] <- 0
prom.sum$g2.c[prom.sum$vote17!=1] <- 0
prom.sum$g3.c[prom.sum$vote17!=1] <- 0
prom.sum$g4.c[prom.sum$vote17!=1] <- 0


prom.sum$mii1 <- pf$agreed_cat[match(prom.sum$g1, pf$id)]
prom.sum$mii2 <- pf$agreed_cat[match(prom.sum$g2, pf$id)]
prom.sum$mii3 <- pf$agreed_cat[match(prom.sum$g3, pf$id)]
prom.sum$mii4 <- pf$agreed_cat[match(prom.sum$g4, pf$id)]

prom.sum$mii1.c <- prom.sum$mii1
prom.sum$mii2.c <- prom.sum$mii2
prom.sum$mii3.c <- prom.sum$mii3
prom.sum$mii4.c <- prom.sum$mii4


prom.sum$mii1.c[prom.sum$vote17!=1] <- 0
prom.sum$mii2.c[prom.sum$vote17!=1] <- 0
prom.sum$mii3.c[prom.sum$vote17!=1] <- 0
prom.sum$mii4.c[prom.sum$vote17!=1] <- 0

prom.sum$votecon <- as.numeric(prom.sum$vote17==1)
prom.sum$weight <- promise$Weight[match(prom.sum$id, promise$ID)]

save(prom.sum, file = "data/intermediate/prom_sum.rda")
write.csv(prom.sum, file = "data/intermediate/stackedData.csv", row.names = F)

promise.index <- promise %>% select(issue = g1r1, text = g1r1p1text) %>%
  unique %>% arrange(issue) %>% dtf

save(promise.index, file = "data/intermediate/promise.index.rda")


#### Output ####
# footnote 1: 
# however, only 4% of respondents gave more than one of the six bundles 
# they were shown a ten, 
prop.table(table((ceilings1+ceilings2)>1))


# with only around 3% giving the maximum score to two or more bundles 
# on the same screen. 
prop.table(table((ceilings1>1) | (ceilings2>1)))



# still not much higher than the 5% of election pledges that were about the EU
prop.table(table(pf$agreed_cat_string))["europe"]
